BDA/IDAR Course Work 2

Adam Russell, 12711142, MSc Data Science (Part time)

1. Decision Trees

(a)

(a)

(b)

(b)

2. Regression Trees

library(MASS)
## Warning: package 'MASS' was built under R version 3.5.2
library(ISLR)
library(tree)
## Warning: package 'tree' was built under R version 3.5.2
fix(Carseats)
?Carseats
## starting httpd help server ... done
set.seed(1)

#(a)

Carseats_split <- sample(1:nrow(Carseats), nrow(Carseats)/2)
Carseats.train <- Carseats[Carseats_split,]
Carseats.test <- Carseats[-Carseats_split,]

#(b)
tree.Carseats <- tree(Carseats$Sales ~ .,data = Carseats, subset = Carseats_split)
summary(tree.Carseats)
## 
## Regression tree:
## tree(formula = Carseats$Sales ~ ., data = Carseats, subset = Carseats_split)
## Variables actually used in tree construction:
## [1] "ShelveLoc"   "Price"       "Age"         "Advertising" "Income"     
## [6] "CompPrice"  
## Number of terminal nodes:  18 
## Residual mean deviance:  2.36 = 429.5 / 182 
## Distribution of residuals:
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
## -4.2570 -1.0360  0.1024  0.0000  0.9301  3.9130
#The tree function has only used: shelve location, price, average age of location population, local advertising budget
#community income and competitor price at each location to model indicating that education level, urban or rural, US or not
#and population size in region are not significant in relation to unit sales of car seats.

plot(tree.Carseats)
text(tree.Carseats,pretty = 0)

#The plot of the regression decision tree indicates a good shelve location to be the biggest driver of unit sales of carseats.
#The second critical driver is Carseat price.

yhat <- predict(tree.Carseats, newdata = Carseats[-Carseats_split,])
Carseats.test_sales <- Carseats[-Carseats_split, "Sales"]
mean((yhat-Carseats.test_sales)^2) #test MSE
## [1] 4.148897
#(c)
cv.Carseats <- cv.tree(tree.Carseats)
plot(cv.Carseats$size, cv.Carseats$dev,type = 'b')

#The CV plot indicates that the minimum MSE is when the number of lead nodes is 7.

prune.Carseats <- prune.tree(tree.Carseats,best=7)

plot(prune.Carseats)
text(prune.Carseats,pretty = 0)

yhat_prune <- predict(prune.Carseats, newdata = Carseats[-Carseats_split,])
mean((yhat_prune-Carseats.test_sales)^2) #test MSE
## [1] 5.340397
#there for prunning the tree to 7 terminal nodes from 18 did not reduce the test MSE indicating the model
#has high bias.

#(d)
library(randomForest)
## Warning: package 'randomForest' was built under R version 3.5.2
## randomForest 4.6-14
## Type rfNews() to see new features/changes/bug fixes.
bag.Carseats = randomForest(Carseats$Sales ~ .,data = Carseats, subset = Carseats_split, mtry = 10, importance = TRUE)
bag.Carseats
## 
## Call:
##  randomForest(formula = Carseats$Sales ~ ., data = Carseats, mtry = 10,      importance = TRUE, subset = Carseats_split) 
##                Type of random forest: regression
##                      Number of trees: 500
## No. of variables tried at each split: 10
## 
##           Mean of squared residuals: 2.811039
##                     % Var explained: 63.17
yhat_bag <- predict(bag.Carseats, newdata = Carseats[-Carseats_split,])
mean((yhat_bag-Carseats.test_sales)^2) #test MSE
## [1] 2.633915
#Large reduction in test MSE to 2.633915 when compared with pruned tree

importance(bag.Carseats)
##                %IncMSE IncNodePurity
## CompPrice   16.9874366    126.852848
## Income       3.8985402     78.314126
## Advertising 16.5698586    123.702901
## Population   0.6487058     62.328851
## Price       55.3976775    514.654890
## ShelveLoc   42.7849818    319.133777
## Age         20.5135255    185.582077
## Education    3.4615211     42.253410
## Urban       -2.5125087      8.700009
## US           7.3586645     18.180651
varImpPlot(bag.Carseats)

#We can see that the biggest drop in prediction accuracy is when the variables Price and Shelve Location are excluded
#which was observed in (b)

#(e)

testMSE <- rep(0,10)
for (i in 1:10) {

set.seed(1)  
rf.Carseats = randomForest(Carseats$Sales ~ .,data = Carseats, subset = Carseats_split, mtry = i, importance = TRUE)
yhat_rf <- predict(rf.Carseats, newdata = Carseats[-Carseats_split,])
testMSE[i] <- mean((yhat_rf-Carseats.test_sales)^2) #test MSE

}

plot(testMSE, type = "b", xlab = "mtry", ylab = "Test MSE")

#The above plot demonstrates that the random forest model is no better then the bagged model suggesting
#the variables of the careseat data set are not highly correlated.


rf.Carseats = randomForest(Carseats$Sales ~ .,data = Carseats, subset = Carseats_split, mtry = 8, importance = TRUE)
importance(rf.Carseats)
##                 %IncMSE IncNodePurity
## CompPrice   13.95307865    130.822017
## Income       5.01601324     80.900674
## Advertising 12.80534252    128.915509
## Population  -0.09912927     66.842641
## Price       54.75918061    496.543261
## ShelveLoc   42.94342059    309.717773
## Age         20.95271103    194.037615
## Education    2.27870853     42.731861
## Urban       -2.07631988      9.806819
## US           6.41852889     18.554327
varImpPlot(rf.Carseats)

#And accordingly the variable importance conclusions for the random forest model are the same as for the bagged
#model in part (d).

3. Classification Trees

library(MASS)
library(ISLR)
library(tree)
library(randomForest)
fix(OJ)
?OJ
set.seed(1)

#(a)

OJ_split <- sample(1:nrow(OJ), 800)
OJ.train <- OJ[OJ_split,]
OJ.test <- OJ[-OJ_split,]

#(b)

tree.OJ <- tree(Purchase ~., data=OJ, subset = OJ_split)
summary(tree.OJ)
## 
## Classification tree:
## tree(formula = Purchase ~ ., data = OJ, subset = OJ_split)
## Variables actually used in tree construction:
## [1] "LoyalCH"       "PriceDiff"     "SpecialCH"     "ListPriceDiff"
## Number of terminal nodes:  8 
## Residual mean deviance:  0.7305 = 578.6 / 792 
## Misclassification error rate: 0.165 = 132 / 800
#The summary indicates that only customer brand loyalty, sale price difference between MM and CH, Sale price for CH and List price difference
#are signifcant predictors In CH or MM purchase. There are 8 terminal nodes. The training error rate is 0.165 = 132 / 800.

#(c)

tree.OJ
## node), split, n, deviance, yval, (yprob)
##       * denotes terminal node
## 
##  1) root 800 1064.00 CH ( 0.61750 0.38250 )  
##    2) LoyalCH < 0.508643 350  409.30 MM ( 0.27143 0.72857 )  
##      4) LoyalCH < 0.264232 166  122.10 MM ( 0.12048 0.87952 )  
##        8) LoyalCH < 0.0356415 57   10.07 MM ( 0.01754 0.98246 ) *
##        9) LoyalCH > 0.0356415 109  100.90 MM ( 0.17431 0.82569 ) *
##      5) LoyalCH > 0.264232 184  248.80 MM ( 0.40761 0.59239 )  
##       10) PriceDiff < 0.195 83   91.66 MM ( 0.24096 0.75904 )  
##         20) SpecialCH < 0.5 70   60.89 MM ( 0.15714 0.84286 ) *
##         21) SpecialCH > 0.5 13   16.05 CH ( 0.69231 0.30769 ) *
##       11) PriceDiff > 0.195 101  139.20 CH ( 0.54455 0.45545 ) *
##    3) LoyalCH > 0.508643 450  318.10 CH ( 0.88667 0.11333 )  
##      6) LoyalCH < 0.764572 172  188.90 CH ( 0.76163 0.23837 )  
##       12) ListPriceDiff < 0.235 70   95.61 CH ( 0.57143 0.42857 ) *
##       13) ListPriceDiff > 0.235 102   69.76 CH ( 0.89216 0.10784 ) *
##      7) LoyalCH > 0.764572 278   86.14 CH ( 0.96403 0.03597 ) *
#Node 21 is a terminal node:
# 21) SpecialCH > 0.5 13   16.05 CH ( 0.69231 0.30769 ) *
#All the observations on this branch have a 'Indicator of special on CH' greater then 0.5.
#There are 13 oberservations on this branch.
#deviance = 16.05
#The overall prediction for this branch is a purchase of CH as indicated by 0.69231 of the observations at this node.

#(d)

plot(tree.OJ)
text(tree.OJ,pretty = 0)

#The resulting classification decision tree indicates that customers with a CH brand loyalty greater then 0.508643 are highly likely to purchase
#CH over MM. This is indicated by ALL the terminal nodes following LoyalCH < 0.508643 split along the NO branch predicting CH purchases. Conversely customers
#with CH brand loyalty less then 0.264232 are highley likely to purchase MM.This is indicated by ALL the terminal nodes following 
#LoyalCH < 0.264232 split along the YES branch predicting MM purchases.Customers with LoyalCH between 0.26232 and 0.508643 make a decision between MM and CH
#based on price difference and if there is a special deal on CH.

#(e)
tree.OJ.pred <- predict(tree.OJ, OJ.test, type = "class")
OJ.test_Purchase <- OJ[-OJ_split, "Purchase"]
table(tree.OJ.pred, OJ.test_Purchase)
##             OJ.test_Purchase
## tree.OJ.pred  CH  MM
##           CH 147  49
##           MM  12  62
(147+62)/270
## [1] 0.7740741
#The test error rate is 0.7740741

#(f)
cv.OJ <- cv.tree(tree.OJ, FUN=prune.misclass)
names(cv.OJ)
## [1] "size"   "dev"    "k"      "method"
cv.OJ
## $size
## [1] 8 5 2 1
## 
## $dev
## [1] 156 156 156 306
## 
## $k
## [1]       -Inf   0.000000   4.666667 160.000000
## 
## $method
## [1] "misclass"
## 
## attr(,"class")
## [1] "prune"         "tree.sequence"
#(g)
par(mfrow=c(1,2))
plot(cv.OJ$size,cv.OJ$dev,type="b")

#(h)
#The results indicate that a classitication decision tree with two or more terminal nodes have lowest error rate.

#(i)
#Tree pruning to 5 terminal nodes was selected as the cross validation classification error is inidcated as being the same
#for 2 or more terminal nodes.
prune.OJ <- prune.misclass(tree.OJ, best = 5)
plot(prune.OJ)
text(prune.OJ,pretty=0)

#(j)
summary(prune.OJ)
## 
## Classification tree:
## snip.tree(tree = tree.OJ, nodes = 3:4)
## Variables actually used in tree construction:
## [1] "LoyalCH"   "PriceDiff" "SpecialCH"
## Number of terminal nodes:  5 
## Residual mean deviance:  0.8256 = 656.4 / 795 
## Misclassification error rate: 0.165 = 132 / 800
#The training error rate is the same as for the unpruned tree: 0.165 = 132/800

#(k)
prune.OJ.pred <- predict(prune.OJ, OJ.test, type = "class")
table(prune.OJ.pred, OJ.test_Purchase)
##              OJ.test_Purchase
## prune.OJ.pred  CH  MM
##            CH 147  49
##            MM  12  62
(147+62)/270
## [1] 0.7740741
#The test error rate is the same as for the unpruned tree: 0.7740741

4.SVM

library(MASS)
library(ISLR)
library(e1071)
## Warning: package 'e1071' was built under R version 3.5.2
library(LiblineaR)
## Warning: package 'LiblineaR' was built under R version 3.5.2
set.seed(1)
fix(Auto)
?Auto

#(a)
above_median_mpg <- ifelse(Auto$mpg<=median(Auto$mpg),0,1)
median(Auto$mpg)
## [1] 22.75
Auto2 <- data.frame(xx=Auto[0:6], yy=as.factor(above_median_mpg))
#Removing name, year of manufacture, cylinders and origin information from data Auto dataset.
head(Auto2)
##   xx.mpg xx.cylinders xx.displacement xx.horsepower xx.weight
## 1     18            8             307           130      3504
## 2     15            8             350           165      3693
## 3     18            8             318           150      3436
## 4     16            8             304           150      3433
## 5     17            8             302           140      3449
## 6     15            8             429           198      4341
##   xx.acceleration yy
## 1            12.0  0
## 2            11.5  0
## 3            11.0  0
## 4            12.0  0
## 5            10.5  0
## 6            10.0  0
#(b)
Auto2.svm.tune.linear <- tune(svm, yy~., data=Auto2, kernel="linear", ranges=list(cost=c(0.001 , 0.01 , 0.1, 1 ,5 ,10 ,100, 1000)))
summary(Auto2.svm.tune.linear)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost
##     5
## 
## - best performance: 0.0025 
## 
## - Detailed performance results:
##    cost       error  dispersion
## 1 1e-03 0.122692308 0.054156614
## 2 1e-02 0.081730769 0.055230186
## 3 1e-01 0.053589744 0.045888164
## 4 1e+00 0.017820513 0.024132627
## 5 5e+00 0.002500000 0.007905694
## 6 1e+01 0.005064103 0.010677135
## 7 1e+02 0.005064103 0.010677135
## 8 1e+03 0.005064103 0.010677135
#The results of the tuning show that the errors are at a minium for a cost equal to 5.

Auto2.svm.linear <- svm(formula = yy~., data = Auto2, kernel = "linear", cost = 5)
summary(Auto2.svm.linear)
## 
## Call:
## svm(formula = yy ~ ., data = Auto2, kernel = "linear", cost = 5)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  linear 
##        cost:  5 
##       gamma:  0.1666667 
## 
## Number of Support Vectors:  25
## 
##  ( 12 13 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  0 1
#(c)
Auto2.svm.tune.radial <- tune(svm, yy~., data=Auto2, kernel="radial", ranges=list(cost=c(0.001 , 0.01 , 0.1, 1 ,5 ,10 ,100, 1000),gamma =c(0.001 , 0.01 , 0.1, 1 ,5 ,10 ,100, 1000)))
summary(Auto2.svm.tune.radial)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost gamma
##  1000 0.001
## 
## - best performance: 0.002564103 
## 
## - Detailed performance results:
##     cost gamma       error  dispersion
## 1  1e-03 1e-03 0.558717949 0.050683112
## 2  1e-02 1e-03 0.558717949 0.050683112
## 3  1e-01 1e-03 0.558717949 0.050683112
## 4  1e+00 1e-03 0.097051282 0.061308926
## 5  5e+00 1e-03 0.086794872 0.041988348
## 6  1e+01 1e-03 0.079102564 0.032595382
## 7  1e+02 1e-03 0.030641026 0.010881923
## 8  1e+03 1e-03 0.002564103 0.008108404
## 9  1e-03 1e-02 0.558717949 0.050683112
## 10 1e-02 1e-02 0.558717949 0.050683112
## 11 1e-01 1e-02 0.094487179 0.060390528
## 12 1e+00 1e-02 0.081666667 0.035523079
## 13 5e+00 1e-02 0.056025641 0.035213260
## 14 1e+01 1e-02 0.028012821 0.014346664
## 15 1e+02 1e-02 0.007692308 0.012385792
## 16 1e+03 1e-02 0.007692308 0.012385792
## 17 1e-03 1e-01 0.558717949 0.050683112
## 18 1e-02 1e-01 0.091923077 0.061748218
## 19 1e-01 1e-01 0.081666667 0.035523079
## 20 1e+00 1e-01 0.048461538 0.025333267
## 21 5e+00 1e-01 0.010256410 0.013240969
## 22 1e+01 1e-01 0.007692308 0.012385792
## 23 1e+02 1e-01 0.012756410 0.013447795
## 24 1e+03 1e-01 0.007628205 0.012283816
## 25 1e-03 1e+00 0.558717949 0.050683112
## 26 1e-02 1e+00 0.558717949 0.050683112
## 27 1e-01 1e+00 0.084230769 0.036063920
## 28 1e+00 1e+00 0.030576923 0.033470093
## 29 5e+00 1e+00 0.022884615 0.021941126
## 30 1e+01 1e+00 0.022948718 0.022275167
## 31 1e+02 1e+00 0.030576923 0.023148137
## 32 1e+03 1e+00 0.030576923 0.023148137
## 33 1e-03 5e+00 0.558717949 0.050683112
## 34 1e-02 5e+00 0.558717949 0.050683112
## 35 1e-01 5e+00 0.161025641 0.075794667
## 36 1e+00 5e+00 0.061217949 0.029803288
## 37 5e+00 5e+00 0.066410256 0.030127750
## 38 1e+01 5e+00 0.066410256 0.030127750
## 39 1e+02 5e+00 0.066410256 0.030127750
## 40 1e+03 5e+00 0.066410256 0.030127750
## 41 1e-03 1e+01 0.558717949 0.050683112
## 42 1e-02 1e+01 0.558717949 0.050683112
## 43 1e-01 1e+01 0.556153846 0.053814941
## 44 1e+00 1e+01 0.094551282 0.034593285
## 45 5e+00 1e+01 0.097179487 0.041840204
## 46 1e+01 1e+01 0.097179487 0.041840204
## 47 1e+02 1e+01 0.097179487 0.041840204
## 48 1e+03 1e+01 0.097179487 0.041840204
## 49 1e-03 1e+02 0.558717949 0.050683112
## 50 1e-02 1e+02 0.558717949 0.050683112
## 51 1e-01 1e+02 0.558717949 0.050683112
## 52 1e+00 1e+02 0.466602564 0.095169918
## 53 5e+00 1e+02 0.448717949 0.090453036
## 54 1e+01 1e+02 0.448717949 0.090453036
## 55 1e+02 1e+02 0.448717949 0.090453036
## 56 1e+03 1e+02 0.448717949 0.090453036
## 57 1e-03 1e+03 0.558717949 0.050683112
## 58 1e-02 1e+03 0.558717949 0.050683112
## 59 1e-01 1e+03 0.558717949 0.050683112
## 60 1e+00 1e+03 0.535641026 0.069585458
## 61 5e+00 1e+03 0.533076923 0.072000114
## 62 1e+01 1e+03 0.533076923 0.072000114
## 63 1e+02 1e+03 0.533076923 0.072000114
## 64 1e+03 1e+03 0.533076923 0.072000114
#The results of the tuning show that the errors are at a minium for costs equal to 1000 and gamma equal to 0.001.

Auto2.svm.radial <- svm(formula = yy~., data = Auto2, gamma = 0.001, kernel = "radial", cost = 1000)
summary(Auto2.svm.radial)
## 
## Call:
## svm(formula = yy ~ ., data = Auto2, gamma = 0.001, kernel = "radial", 
##     cost = 1000)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  radial 
##        cost:  1000 
##       gamma:  0.001 
## 
## Number of Support Vectors:  36
## 
##  ( 17 19 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  0 1
Auto2.svm.tune.poly <- tune(svm, yy~., data=Auto2, kernel="polynomial", ranges=list(cost=c(0.001 , 0.01 , 0.1, 1 ,5 ,10 ,100, 1000),degree=c(2,3,4,5)))
summary(Auto2.svm.tune.poly)
## 
## Parameter tuning of 'svm':
## 
## - sampling method: 10-fold cross validation 
## 
## - best parameters:
##  cost degree
##  1000      3
## 
## - best performance: 0.02544872 
## 
## - Detailed performance results:
##     cost degree      error dispersion
## 1  1e-03      2 0.58673077 0.03560377
## 2  1e-02      2 0.45948718 0.09665151
## 3  1e-01      2 0.29314103 0.06439963
## 4  1e+00      2 0.24730769 0.05790701
## 5  5e+00      2 0.24724359 0.07300194
## 6  1e+01      2 0.25237179 0.07077783
## 7  1e+02      2 0.22961538 0.06285452
## 8  1e+03      2 0.19634615 0.06809105
## 9  1e-03      3 0.35211538 0.11377480
## 10 1e-02      3 0.25506410 0.05294702
## 11 1e-01      3 0.22192308 0.06855266
## 12 1e+00      3 0.08416667 0.02420111
## 13 5e+00      3 0.06128205 0.02159748
## 14 1e+01      3 0.04339744 0.02403832
## 15 1e+02      3 0.02807692 0.02537217
## 16 1e+03      3 0.02544872 0.01688453
## 17 1e-03      4 0.43358974 0.10882896
## 18 1e-02      4 0.34942308 0.07545637
## 19 1e-01      4 0.26262821 0.05515474
## 20 1e+00      4 0.25224359 0.06429973
## 21 5e+00      4 0.24455128 0.06928960
## 22 1e+01      4 0.22423077 0.06216796
## 23 1e+02      4 0.25224359 0.04818388
## 24 1e+03      4 0.20141026 0.05538501
## 25 1e-03      5 0.36730769 0.08638189
## 26 1e-02      5 0.28057692 0.06415813
## 27 1e-01      5 0.24993590 0.05252452
## 28 1e+00      5 0.19871795 0.06975702
## 29 5e+00      5 0.13006410 0.02518913
## 30 1e+01      5 0.11730769 0.02733039
## 31 1e+02      5 0.08929487 0.03230957
## 32 1e+03      5 0.05096154 0.03751354
#The results of the tuning show that the errors are at a minium for costs equal to 1000 and degree equal to 3.

Auto2.svm.poly <- svm(formula = yy~., data = Auto2, kernel = "polynomial", cost = 1000, degree = 3)
summary(Auto2.svm.poly)
## 
## Call:
## svm(formula = yy ~ ., data = Auto2, kernel = "polynomial", cost = 1000, 
##     degree = 3)
## 
## 
## Parameters:
##    SVM-Type:  C-classification 
##  SVM-Kernel:  polynomial 
##        cost:  1000 
##      degree:  3 
##       gamma:  0.1666667 
##      coef.0:  0 
## 
## Number of Support Vectors:  35
## 
##  ( 17 18 )
## 
## 
## Number of Classes:  2 
## 
## Levels: 
##  0 1
#(d)

plot(Auto2.svm.linear, Auto2, xx.mpg ~ xx.cylinders)

plot(Auto2.svm.radial, Auto2, xx.mpg ~ xx.cylinders)

plot(Auto2.svm.poly, Auto2, xx.mpg~ xx.cylinders)

plot(Auto2.svm.linear, Auto2, xx.mpg ~ xx.displacement)

plot(Auto2.svm.radial, Auto2, xx.mpg ~ xx.displacement)

plot(Auto2.svm.poly, Auto2, xx.mpg~ xx.displacement)

plot(Auto2.svm.linear, Auto2, xx.mpg ~ xx.horsepower)

plot(Auto2.svm.radial, Auto2, xx.mpg ~ xx.horsepower)

plot(Auto2.svm.poly, Auto2, xx.mpg~ xx.horsepower)

plot(Auto2.svm.linear, Auto2, xx.mpg ~ xx.weight)

plot(Auto2.svm.radial, Auto2, xx.mpg ~ xx.weight)

plot(Auto2.svm.poly, Auto2, xx.mpg~ xx.weight)

plot(Auto2.svm.linear, Auto2, xx.mpg ~ xx.acceleration)

plot(Auto2.svm.radial, Auto2, xx.mpg ~ xx.acceleration)

plot(Auto2.svm.poly, Auto2, xx.mpg~ xx.acceleration)

5.SVM

library(MASS)
library(ISLR)
library(e1071)
library(LiblineaR)
set.seed(1)

#(a)
X1 <- c(3, 2, 4, 1, 2, 4, 4)
X2 <- c(4, 2, 4, 4, 1, 3, 1)

#Plot function graphical parameters: 2 = red, 4 = blue
Y <- c(2,2,2,2,4,4,4)

toy=data.frame(X1,X2,Y)

plot(toy$X1, toy$X2,col = Y)

#(b)
#The data frame 'toy' has two predictor dimensions so it may be possible to define the maximal margin hyperplane
#by defining the linear equation of a line sperating the two classes.
#By eye, we can see that observations 2 & 3 and 5 & 6 are logical support vectors for a hyperplane.
#The limits of the hyperplane margin are defined by lines that pass over observations 2 & 3 and 5 & 6.
#Both these lines have equal linear gradients:
# m = dX2/dX1 = 4-2 / 4-2 = 1 for observations 2 & 3
# m = dX2/dX1 = 3-1 / 4-2 = 1 for observations 5 & 6
#There for the linear gradient of the hyperplane line is also 1.
#The linear intercepts of these marging limits are as follows:
# y-4 = m(x-4) for observation 2 leading to a upper margin line defined by the linear equation X2=X1
# y-3 = m(x-4) for observation 6 leading to a lower margin line defined by the linear equation x2=X1-1
#There for the linear intercept of the hyperplane line is -1/2 = -0.5.

abline(a=-0.5,b=1, col = 3)

#There for the hyperplane line is defined by
#-0.5 + X1 - X2 = 0
#hyper plane coefficients: BETA0=-0.5, BETA1=1, BETA2=-1

#(c)
#classify to red if [-0.5 + X1 - X2 < 0] and classify to blue otherwise.
-0.5+toy$X1-toy$X2
## [1] -1.5 -0.5 -0.5 -3.5  0.5  0.5  2.5
#(d)
abline(a=0,b=1, col = 5)
abline(a=-1,b=1, col = 5)

#(e)
#The support vectors are observations 2 & 3 and 5 & 6.

#(e)
#As obvservation 7 is no a support vector, a slight movement such that it does not approach a margin will not impact
#the maximal margin hyper plane.

#(f)
#An example of a non optimal seperating hyper plane has the following equation and is plotted in red:
#[-1 + 1.2*X1 -X2 = 0]
abline(a=-1, b=1.2, col = 6)

#(g)
#An example of an additional Blue point [+] that would make the two classes no longer seperable by a hyperplane
#is shown at position (2, 3.5):
points(2,3.5, col = 4, pch = "+")

6. Hierarchical Clustering

library(MASS)
library(ISLR)
set.seed(1)
fix(USArrests)
?USArrests

#(a)
hc.complete <- hclust(dist(USArrests), method ="complete")
plot(hc.complete, main ='Complete', cex=0.9)

#(b)
cutree(hc.complete, k=3)
##        Alabama         Alaska        Arizona       Arkansas     California 
##              1              1              1              2              1 
##       Colorado    Connecticut       Delaware        Florida        Georgia 
##              2              3              1              1              2 
##         Hawaii          Idaho       Illinois        Indiana           Iowa 
##              3              3              1              3              3 
##         Kansas       Kentucky      Louisiana          Maine       Maryland 
##              3              3              1              3              1 
##  Massachusetts       Michigan      Minnesota    Mississippi       Missouri 
##              2              1              3              1              2 
##        Montana       Nebraska         Nevada  New Hampshire     New Jersey 
##              3              3              1              3              2 
##     New Mexico       New York North Carolina   North Dakota           Ohio 
##              1              1              1              3              3 
##       Oklahoma         Oregon   Pennsylvania   Rhode Island South Carolina 
##              2              2              3              2              1 
##   South Dakota      Tennessee          Texas           Utah        Vermont 
##              3              2              2              3              3 
##       Virginia     Washington  West Virginia      Wisconsin        Wyoming 
##              2              2              3              3              2
#(c)
apply(USArrests , 2, mean )
##   Murder  Assault UrbanPop     Rape 
##    7.788  170.760   65.540   21.232
apply(USArrests , 2, var )
##     Murder    Assault   UrbanPop       Rape 
##   18.97047 6945.16571  209.51878   87.72916
USArrests.scaled <- scale(USArrests, scale = TRUE)
apply(USArrests.scaled , 2, mean )
##        Murder       Assault      UrbanPop          Rape 
## -7.663087e-17  1.112408e-16 -4.332808e-16  8.942391e-17
apply(USArrests.scaled , 2, var )
##   Murder  Assault UrbanPop     Rape 
##        1        1        1        1
hc.complete.scaled <- hclust(dist(USArrests.scaled), method ="complete")
plot(hc.complete.scaled, main ='Complete', cex=0.9)

cutree(hc.complete.scaled, k=3)
##        Alabama         Alaska        Arizona       Arkansas     California 
##              1              1              2              3              2 
##       Colorado    Connecticut       Delaware        Florida        Georgia 
##              2              3              3              2              1 
##         Hawaii          Idaho       Illinois        Indiana           Iowa 
##              3              3              2              3              3 
##         Kansas       Kentucky      Louisiana          Maine       Maryland 
##              3              3              1              3              2 
##  Massachusetts       Michigan      Minnesota    Mississippi       Missouri 
##              3              2              3              1              3 
##        Montana       Nebraska         Nevada  New Hampshire     New Jersey 
##              3              3              2              3              3 
##     New Mexico       New York North Carolina   North Dakota           Ohio 
##              2              2              1              3              3 
##       Oklahoma         Oregon   Pennsylvania   Rhode Island South Carolina 
##              3              3              3              3              1 
##   South Dakota      Tennessee          Texas           Utah        Vermont 
##              3              1              2              3              3 
##       Virginia     Washington  West Virginia      Wisconsin        Wyoming 
##              3              3              3              3              3
#(d)
plot(hc.complete, main ='Complete', cex=0.9)

plot(hc.complete.scaled, main ='Complete', cex=0.9)

#Observing the denograms with unscaled variables on the left and scaled variable with SD = 1 on the right, the right hand
#plot has more gradual levels representing where all the parameters of the data set have equal impact on the clustering algorithm.
#Alaska stands out on the scaled dendrogram because of its low urban population yet comparable arrest rates to states with
#much higher urban populaions.

7. PCA & K-Means Clustering

library(MASS)
library(ISLR)
set.seed(19)

#(a)
x = matrix(rnorm(60*50), ncol = 50)
x[1:20,]= x[1:20,]+2.4 #CLUSTER A
x[21:40,]= x[21:40,]+0.5 #CLuSTER B
x[41:60,]= x[41:60,]+1.2 #CLUSTER C
k <- c(rep('CLUSTER A', times=20), rep('CLUSTER B', times=20), rep('CLUSTER C', times=20))
k_col <- c(rep(2, times=20), rep(3, times=20), rep(4, times=20))

#(b)
pca.x <- prcomp(x)
pca.x
## Standard deviations (1, .., p=50):
##  [1] 5.70043345 1.73316723 1.68258007 1.64441414 1.59386890 1.55148592
##  [7] 1.51289350 1.47090301 1.41745951 1.37640139 1.35034907 1.31500472
## [13] 1.27505467 1.24467602 1.14904670 1.12918122 1.09922696 1.05619019
## [19] 1.03675005 1.02587138 0.96064811 0.93903240 0.87976089 0.87048887
## [25] 0.83701095 0.81944756 0.79535198 0.77053575 0.71878544 0.70192274
## [31] 0.69333026 0.66626286 0.63229137 0.60818074 0.54963197 0.52269952
## [37] 0.48695590 0.46216902 0.44088277 0.40994136 0.35660038 0.33472843
## [43] 0.31936478 0.26555329 0.20859858 0.19359866 0.17547798 0.14425067
## [49] 0.08876469 0.06981792
## 
## Rotation (n x k) = (50 x 50):
##              PC1          PC2          PC3          PC4          PC5
##  [1,] 0.14315202  0.020569318 -0.110037211  0.176223585 -0.013850384
##  [2,] 0.12610636 -0.233011591  0.081330827 -0.190910191  0.164258986
##  [3,] 0.19276192  0.144821992 -0.187176867 -0.191588589 -0.016114880
##  [4,] 0.17508489 -0.104132981 -0.169353613  0.320467109 -0.175868346
##  [5,] 0.12719038 -0.002095026  0.014385494  0.067962499 -0.131620600
##  [6,] 0.16141878  0.201024647  0.222315842 -0.106961667  0.082123819
##  [7,] 0.09119935 -0.010925516  0.146180415 -0.088137740  0.082365978
##  [8,] 0.16303827 -0.159628827  0.072397828 -0.011547914  0.143577200
##  [9,] 0.14297033 -0.036925263 -0.012993590 -0.027589134 -0.105775147
## [10,] 0.16432252  0.303688856  0.117536878 -0.187889346  0.021600310
## [11,] 0.12887360 -0.074900284  0.179853816 -0.191772256 -0.160226121
## [12,] 0.17269567  0.209227555 -0.026796362  0.238360376 -0.038284366
## [13,] 0.12129558  0.167127150 -0.266775155 -0.004101417 -0.045278392
## [14,] 0.17142903  0.037704170  0.109553781  0.265787845 -0.003459431
## [15,] 0.13633103 -0.081741864  0.076192438  0.154951185  0.233203829
## [16,] 0.14123974  0.057624887 -0.021150634 -0.124315319 -0.179463752
## [17,] 0.10895098  0.241837720 -0.046252580 -0.105773245  0.031728009
## [18,] 0.16407555 -0.145627590  0.138200881 -0.044637973 -0.114437734
## [19,] 0.14932051 -0.074479455  0.071822847 -0.172991286 -0.196662644
## [20,] 0.16638833  0.114761082 -0.129651853  0.036279546 -0.342219001
## [21,] 0.14918036  0.046131746 -0.033406536  0.193789785  0.133407808
## [22,] 0.14898155 -0.043202224 -0.106143983 -0.134896254  0.201739867
## [23,] 0.09638570 -0.002337495  0.060322740  0.122717682  0.051021264
## [24,] 0.14437935  0.004788681  0.106079545 -0.005330893 -0.089222467
## [25,] 0.14661390 -0.094135587  0.007600119  0.038628138  0.301591423
## [26,] 0.13671053  0.295459712 -0.201019355 -0.305116512  0.160077291
## [27,] 0.14322091 -0.253266509  0.196285103  0.136939847 -0.179864236
## [28,] 0.13050584  0.022718508  0.103478777  0.078671462 -0.090108871
## [29,] 0.17092806 -0.223966737 -0.113599841 -0.092763240 -0.071662490
## [30,] 0.12049760  0.011471902  0.077199840 -0.001897891 -0.214078316
## [31,] 0.09502931  0.238753990 -0.022808505  0.293204198 -0.038298084
## [32,] 0.12832383  0.146764038  0.330299087  0.175777727  0.335802577
## [33,] 0.10925551 -0.153282648  0.081203687 -0.079907024 -0.097412231
## [34,] 0.17023087 -0.029582867  0.107209463 -0.014883440  0.080463564
## [35,] 0.11466515  0.206276807  0.225824045 -0.046087585 -0.020266200
## [36,] 0.13421267 -0.196452780 -0.317732216 -0.164833378  0.103173203
## [37,] 0.14229398  0.161373039  0.091865952  0.080257529  0.006901402
## [38,] 0.16069967 -0.028831727 -0.107791233  0.057940769  0.136550947
## [39,] 0.15886617 -0.015258188 -0.184992063  0.111129929  0.137852289
## [40,] 0.15009341 -0.048557029 -0.111151232  0.059002523 -0.161052715
## [41,] 0.15241599 -0.146950623 -0.287055737  0.125131305  0.064642543
## [42,] 0.13081344 -0.050500085 -0.054157488 -0.097552782  0.202200268
## [43,] 0.11573324 -0.033674275  0.086352779 -0.144012178  0.053502621
## [44,] 0.12874929  0.161957178 -0.098335239 -0.102708915 -0.188653865
## [45,] 0.16650592 -0.253662511  0.197153207 -0.053836235 -0.059557400
## [46,] 0.10886248  0.033679002 -0.029160394 -0.065095683  0.036937932
## [47,] 0.10658892 -0.113050917 -0.127998117 -0.072286835  0.070884212
## [48,] 0.11929440 -0.053540654 -0.070949712  0.152964563  0.076258968
## [49,] 0.14503044 -0.002879098  0.115893453 -0.116778355 -0.093399551
## [50,] 0.09694950 -0.073549253 -0.049789417 -0.041166770  0.053646635
##                 PC6           PC7          PC8          PC9         PC10
##  [1,]  0.1904537451 -0.2360331739  0.061576387 -0.002452703 -0.173595556
##  [2,]  0.0694763492  0.0997616423 -0.067910245 -0.035521972  0.007116069
##  [3,] -0.0774493344  0.0690004381  0.055008835 -0.088256359  0.112084477
##  [4,]  0.0586929555  0.0380847369 -0.085682111  0.095303711 -0.099713299
##  [5,] -0.1594777679 -0.0584208216 -0.235119532  0.054096132 -0.239552189
##  [6,]  0.1727984341 -0.1138784188 -0.270278930 -0.028433311  0.035900749
##  [7,] -0.0315285710  0.0999534916 -0.002548205  0.066786678  0.215746984
##  [8,] -0.1982293758 -0.5148117619 -0.011954937  0.040656602  0.044462648
##  [9,] -0.0359776054  0.0285142771  0.011661746  0.012539917  0.226372709
## [10,]  0.1255347954  0.0394735239  0.040895818 -0.343459593 -0.170131286
## [11,]  0.1319628050  0.0222503331  0.034749117 -0.041685613  0.105558768
## [12,] -0.1452298203  0.0427152036  0.015185430 -0.083439346 -0.284464236
## [13,]  0.0514924039 -0.0300728002  0.048459997 -0.089386022  0.040641306
## [14,] -0.2308469828 -0.0398915542  0.080778842  0.172287052  0.283531275
## [15,]  0.1963295996  0.1005885646 -0.026558178 -0.028063874  0.194186980
## [16,] -0.0833236661 -0.1420916460  0.233400634  0.166784501  0.016422236
## [17,] -0.1903803760 -0.0561881541  0.010645607  0.204719117  0.210220176
## [18,]  0.2823489885  0.0192829952  0.019750988  0.115648255 -0.025788603
## [19,] -0.0853912707  0.1931812116  0.228454725  0.109747133  0.165370024
## [20,]  0.1359490062 -0.0864227367  0.158455067 -0.327060383  0.222334798
## [21,]  0.0382571237 -0.0597317067 -0.079170042  0.112229801  0.059665795
## [22,] -0.1840315529 -0.1383081662  0.202036494 -0.168246262 -0.034894466
## [23,] -0.2610817390  0.2403475894 -0.267808132 -0.301096172  0.045357797
## [24,]  0.1804013049  0.0448094525  0.293381854  0.224288682 -0.169654795
## [25,]  0.0270141061  0.2655817077  0.094882672 -0.107849951 -0.046128289
## [26,] -0.1391046839  0.0445957911 -0.065232727  0.200172362 -0.139058829
## [27,] -0.0343057382 -0.0861631453  0.002893815 -0.170516662  0.017928189
## [28,] -0.0062290420  0.0978389005 -0.001971884  0.206563686 -0.076208645
## [29,] -0.0416538795  0.0686575611 -0.173193130 -0.055974764 -0.108300817
## [30,]  0.0020223111 -0.0791596731  0.084175481  0.001579757 -0.069901992
## [31,] -0.0071931678  0.2174398695  0.031811302 -0.036573681  0.151166321
## [32,] -0.0073700930 -0.0232574522  0.257054450  0.039375512 -0.114135671
## [33,] -0.2718846200  0.0772790888 -0.325386305  0.154269961  0.041345419
## [34,]  0.0547585475 -0.0176557302  0.080657495  0.083933715 -0.324843512
## [35,]  0.1051989418 -0.0336398340 -0.225529747 -0.099501336  0.260409493
## [36,]  0.0018984980  0.1365310402  0.189611040 -0.106737864  0.026960187
## [37,] -0.2043037047 -0.0201718092  0.036314791 -0.090986021  0.022430940
## [38,]  0.2728038609 -0.0527120981 -0.226633363  0.011255042 -0.024543472
## [39,] -0.0391764101 -0.0944281322 -0.061483307  0.139170101  0.052610873
## [40,]  0.1611130402  0.0551614358 -0.084705924  0.079570497  0.090657281
## [41,]  0.0281175666  0.0004370221  0.028338000 -0.072926583  0.019860947
## [42,]  0.0413360716  0.0944168642  0.027573357  0.206268521  0.145286743
## [43,]  0.2892060181 -0.1072242430 -0.094649524  0.089108587  0.060152552
## [44,]  0.0001172125 -0.0268541350 -0.173951793  0.218648345 -0.155420709
## [45,] -0.2588279252 -0.0848083296  0.106156329 -0.142990182 -0.095033999
## [46,]  0.0444695454 -0.2298308969 -0.115087457 -0.276629115 -0.031543674
## [47,] -0.0200712431 -0.0355755723 -0.202842019  0.006685687 -0.006848802
## [48,]  0.0591816770  0.2012899767  0.070642298  0.031946767  0.024590897
## [49,] -0.0599175756  0.3729005430 -0.041608581 -0.038913817 -0.248475990
## [50,] -0.0414515869 -0.1301194426  0.096569309 -0.062256318  0.030587075
##               PC11         PC12          PC13        PC14         PC15
##  [1,]  0.096241774 -0.144023688  0.3486490418 -0.02165015 -0.072884370
##  [2,]  0.208572291  0.250220204 -0.1632291619 -0.39343672 -0.041443149
##  [3,]  0.301141010 -0.065149629 -0.0367805107 -0.09811919 -0.278334067
##  [4,]  0.085156657  0.271302724  0.2049740543 -0.12824101  0.185712784
##  [5,] -0.165972518 -0.050683035 -0.2380655870  0.13264287 -0.114571519
##  [6,] -0.009586066  0.098441865 -0.0977156091  0.08577415 -0.036655738
##  [7,] -0.113568448  0.194428781  0.1109203772  0.09312707  0.019318064
##  [8,] -0.032769937 -0.059280963 -0.0524929003 -0.26057392 -0.077614753
##  [9,]  0.011849542  0.205159879  0.0767659740 -0.01583086  0.002457592
## [10,]  0.123254405  0.134593177  0.0794626873 -0.09660302 -0.017513424
## [11,]  0.034966877 -0.345788008  0.2555637810  0.09346358  0.334043935
## [12,]  0.030972523  0.030734144 -0.0503256114  0.08540928 -0.102491001
## [13,] -0.098696789 -0.116329869  0.0470878191  0.04301613 -0.109764912
## [14,] -0.099159988 -0.097942569 -0.1977061311  0.13646308  0.040014358
## [15,] -0.137410206  0.100659480 -0.0022379293  0.14663716 -0.193708023
## [16,]  0.095369169 -0.072476697  0.1187171284 -0.07993014  0.024091178
## [17,] -0.100263330  0.114796250  0.1349740586  0.03554168 -0.098908118
## [18,] -0.009030159 -0.053377513 -0.0251707225  0.14117684 -0.286445219
## [19,]  0.063437575 -0.020288534  0.0717641971  0.19852343 -0.222421918
## [20,] -0.163660115 -0.047128064 -0.2727646548 -0.20570426  0.037845803
## [21,]  0.199915065 -0.310308763 -0.1618238645  0.15171767 -0.010048096
## [22,] -0.079881383  0.153031491 -0.0561460483  0.01924158 -0.027035003
## [23,] -0.049885878  0.028108365  0.1347798222  0.07229026  0.014668947
## [24,]  0.106418359  0.099394335 -0.0512159656  0.03011886  0.033564915
## [25,]  0.174513227 -0.120321504 -0.1010485214  0.11663825  0.096614497
## [26,] -0.052129755 -0.077018188 -0.0320306972 -0.08514803  0.211655455
## [27,]  0.048667208 -0.112743492 -0.1760647190 -0.05398538  0.125610190
## [28,] -0.117541813  0.151148156  0.1924439841  0.10988441 -0.177261636
## [29,]  0.185868818 -0.094403516 -0.0880157069  0.13958062  0.118306642
## [30,]  0.038241820  0.299460397 -0.2644111260  0.16944709  0.224460889
## [31,] -0.176855175 -0.211087582 -0.0693443971 -0.33733967  0.014196271
## [32,] -0.131063335 -0.020409823  0.1340394904 -0.09372747  0.049464022
## [33,] -0.102514438 -0.028220688  0.1614640054 -0.20150586  0.022743877
## [34,] -0.244910553 -0.102550399 -0.0527780637 -0.09822019 -0.024767363
## [35,]  0.142684816 -0.169967444  0.1573967326 -0.01459822  0.038282306
## [36,] -0.403536044 -0.063261854  0.0829200021  0.07984549  0.189299958
## [37,]  0.204890983  0.204579923 -0.0572148873  0.17031404  0.233050690
## [38,] -0.056237386  0.044083146  0.0126329447  0.03337905  0.175434223
## [39,]  0.305760504  0.048734994  0.0881050987 -0.13487203 -0.053463232
## [40,] -0.116876426  0.245557402  0.0342562102 -0.15254071 -0.076530573
## [41,]  0.148193319 -0.055301715  0.1681807117  0.17293906 -0.055136063
## [42,]  0.023382864  0.007840987 -0.1802880708  0.06589222  0.252001303
## [43,] -0.131253166 -0.067031934 -0.0831491534 -0.08746111  0.017019524
## [44,] -0.053348389 -0.053127123 -0.0557832797  0.06843266  0.171159447
## [45,] -0.016127731 -0.081827990  0.2214588086 -0.07516022  0.038544727
## [46,] -0.208214162  0.155907816  0.1706436705  0.23837294  0.014871879
## [47,] -0.108264010 -0.091454474 -0.0947399413  0.02599144 -0.241645231
## [48,] -0.053351170  0.047027716  0.0006445563 -0.19927045  0.103491572
## [49,] -0.030728184 -0.121110249 -0.0591768829 -0.09856165 -0.200543753
## [50,]  0.002675002  0.008154446 -0.1706696746  0.12857345 -0.182550914
##                PC16         PC17          PC18         PC19         PC20
##  [1,]  0.2796445940  0.055601694 -0.0338521415 -0.182615491  0.109750828
##  [2,] -0.0886176265  0.283246165  0.0708958712  0.038658437  0.007117845
##  [3,]  0.2055237241 -0.189396715  0.0219705236  0.070751717 -0.074466537
##  [4,]  0.0704168434 -0.087934485  0.0103626456 -0.080279387  0.018789983
##  [5,]  0.1436213422  0.107625353  0.2019408700  0.093768075 -0.142271427
##  [6,] -0.0650755512 -0.060631900 -0.1053678680  0.197992084  0.292037287
##  [7,] -0.0041811431 -0.263355581 -0.3548437549 -0.085605434 -0.133747658
##  [8,]  0.2196456111 -0.104820567  0.0951148615 -0.068353094 -0.049726256
##  [9,] -0.0182554336  0.008774002  0.0010885445  0.149295583  0.120096102
## [10,]  0.1530816336 -0.088338092  0.1141639897  0.062551329  0.064179424
## [11,] -0.0002372424  0.093905360  0.1371369214 -0.008052025 -0.023254481
## [12,] -0.0553793317  0.213514056  0.0600232083  0.055582690 -0.158820114
## [13,] -0.0855341019  0.094214609 -0.1097249369  0.099715468  0.165612018
## [14,]  0.1244730872  0.041550844  0.0955446290 -0.068705463  0.153201237
## [15,]  0.1379838735  0.179856120  0.0204136469 -0.113891180  0.054398071
## [16,] -0.1341698758  0.158788371 -0.0341238966  0.015692204 -0.016310058
## [17,] -0.1992020120  0.100626967  0.2114616327 -0.123272918  0.059322905
## [18,]  0.0648994327  0.170149556 -0.3296569259  0.145139614 -0.241550645
## [19,]  0.0523743288 -0.144762788  0.3470028086  0.018348606  0.227178046
## [20,]  0.0287696521  0.034211040 -0.2053420664  0.059141701  0.040731312
## [21,] -0.2409005896 -0.138056989 -0.2405552841  0.076350159  0.120982277
## [22,] -0.0204698420 -0.067104349 -0.0984379037 -0.127947773  0.162769605
## [23,]  0.0674627780  0.082952584 -0.0170814152  0.033117732  0.021325731
## [24,] -0.0489034061 -0.179294182  0.0009124662 -0.203681680 -0.169020081
## [25,] -0.2453280705  0.055939146  0.0071036687 -0.208190392  0.118632170
## [26,] -0.1094730686 -0.029472297 -0.1295771919 -0.043681082  0.024181721
## [27,] -0.2079099106 -0.244876675  0.0676349246 -0.008687854  0.176992193
## [28,] -0.0377794920 -0.007072280 -0.1033316377  0.140766803  0.151375935
## [29,]  0.0739835923 -0.060435966  0.0156818140  0.036236768  0.075200993
## [30,]  0.2517129838  0.057171965 -0.1412826727 -0.109578110  0.084389659
## [31,] -0.0727895306 -0.143786070  0.0108740682  0.068192512 -0.218882329
## [32,]  0.1010927467 -0.167517093 -0.0079422482  0.086286748 -0.080927047
## [33,]  0.0704725346  0.069214718 -0.1710290066  0.068399537  0.054193275
## [34,] -0.0549050711  0.168797791  0.1221763118  0.219991226  0.267979030
## [35,]  0.1259397577  0.204285473  0.1195909006 -0.205549407 -0.158661159
## [36,]  0.1929209074  0.076511853 -0.0149537147  0.042242425 -0.002451679
## [37,] -0.0306268428  0.152974689 -0.0692061145 -0.096627238 -0.133388997
## [38,] -0.0100683218 -0.284162685  0.3426644906  0.076303017 -0.002299501
## [39,] -0.0140456665  0.033833442 -0.1025890676  0.163505435  0.049064123
## [40,] -0.3000971317 -0.126373422  0.1954916100 -0.004272270 -0.132880332
## [41,] -0.0546007927  0.050744656 -0.0314258002  0.146245949 -0.090309992
## [42,]  0.2554664205 -0.055422660  0.0551445139  0.352826621 -0.320508018
## [43,] -0.1214281275  0.180511578 -0.0656281959 -0.140402561  0.010339085
## [44,] -0.0491002813  0.016416192 -0.0146826214 -0.179151418 -0.061961246
## [45,] -0.2261455734 -0.015888476 -0.1273110206  0.072691368 -0.195483289
## [46,] -0.2348009219 -0.107485002  0.0001510426  0.004486426 -0.185729272
## [47,]  0.0743477082 -0.200706833 -0.0860847425 -0.453030561 -0.047554994
## [48,]  0.0091524486  0.236943876  0.0040464013 -0.272768717  0.080044403
## [49,]  0.0683903749 -0.166986167 -0.0422752729 -0.136537203 -0.058131378
## [50,] -0.1964373370  0.163729526  0.2197221194 -0.027364655 -0.314466832
##               PC21         PC22         PC23         PC24         PC25
##  [1,]  0.134775051 -0.144269717 -0.044616669 -0.163477843  0.037333659
##  [2,]  0.068723628  0.103571100  0.033136816 -0.021869522 -0.135159150
##  [3,]  0.055999429 -0.047825567 -0.181042709  0.000433723  0.183546487
##  [4,] -0.057774272 -0.045774325 -0.077618624  0.106838211  0.045184006
##  [5,]  0.031996760  0.276078461  0.061487447  0.080634387 -0.103625726
##  [6,]  0.039342580  0.049616356 -0.028464523 -0.106782978 -0.268269234
##  [7,]  0.196783839  0.060815533  0.295776812 -0.027815915  0.265369249
##  [8,]  0.107446157 -0.060580134  0.146123910 -0.084490313  0.074431661
##  [9,]  0.190802797  0.007573996 -0.129199923 -0.188219163 -0.098636468
## [10,] -0.034182065  0.099527925  0.055174083  0.249318200  0.124051356
## [11,] -0.147251277  0.011680007 -0.109590316 -0.102938854 -0.154745726
## [12,] -0.077796885 -0.232472096  0.213735580 -0.091742095  0.074982900
## [13,] -0.258187732 -0.020436363  0.336588829  0.123766345  0.291249016
## [14,]  0.094389425  0.006015762  0.022968051  0.192415245 -0.016179095
## [15,] -0.356070066 -0.040796044  0.040738371 -0.400956449  0.039733110
## [16,] -0.064161669  0.020912178  0.044095767 -0.080282605 -0.195753247
## [17,]  0.025080146  0.010410168 -0.008383990  0.096710761  0.004929064
## [18,]  0.102212466 -0.073601456 -0.179142024  0.331201080  0.090629650
## [19,] -0.005430026 -0.025630460 -0.021458606 -0.001173090  0.043883100
## [20,] -0.048099439  0.117657111 -0.107880338 -0.107002459 -0.072394341
## [21,] -0.033658184 -0.047197440  0.041541468 -0.075491815 -0.145981723
## [22,] -0.289107254 -0.057998241  0.130616671  0.176172398 -0.292238135
## [23,] -0.135690839  0.242923859 -0.213242811 -0.023878991  0.119458377
## [24,] -0.119244329  0.443396588  0.193138774 -0.177477969 -0.005908400
## [25,]  0.150317311  0.038399314  0.076208892  0.136370600  0.104786535
## [26,]  0.065036966  0.001079783 -0.175663700 -0.133636772  0.133061758
## [27,] -0.103817952 -0.200321169 -0.133539809  0.081119903  0.279422311
## [28,]  0.032610246  0.075225498 -0.162485735  0.103672922 -0.047072310
## [29,]  0.259404261 -0.056800702  0.253511122  0.144674335 -0.167120085
## [30,]  0.022393867  0.011655097  0.053267967 -0.001962173 -0.020253128
## [31,]  0.180795825  0.137608127  0.051530962 -0.036084016 -0.126279141
## [32,]  0.053728141 -0.091980628 -0.165452835  0.184184881 -0.183635052
## [33,] -0.079547316 -0.085158023  0.146867697 -0.051267937  0.031912991
## [34,]  0.143981865  0.119204477  0.045150513 -0.158694204  0.237254644
## [35,]  0.053509930  0.041520089  0.271396655  0.147142545 -0.041415053
## [36,]  0.235860437 -0.039821905 -0.017784076  0.014862645 -0.037491182
## [37,]  0.213572389 -0.146584243 -0.163214750 -0.285564608  0.152081817
## [38,] -0.027663502  0.101004524 -0.056292113 -0.062378287  0.107548661
## [39,] -0.151486544  0.067893459 -0.146982009  0.094552213 -0.035688961
## [40,]  0.043552083 -0.230174153  0.173089870  0.025710762 -0.029942380
## [41,]  0.157810073  0.208717792  0.117741188 -0.105759619 -0.141318861
## [42,] -0.337487226 -0.154901344  0.030599023  0.028004991  0.045414636
## [43,]  0.068202678 -0.112960523 -0.049662901  0.026309659  0.213582434
## [44,] -0.164212988 -0.005167821 -0.130230871  0.139731797 -0.008207584
## [45,] -0.179082992  0.200256230 -0.009403441 -0.005376599  0.111018779
## [46,]  0.021643232 -0.139672700 -0.035676736  0.051124985 -0.191576920
## [47,] -0.055048904  0.212856424 -0.218526235 -0.015698973 -0.036657127
## [48,] -0.071945119 -0.040522228 -0.107346846  0.235132792 -0.093803119
## [49,] -0.047380021 -0.385173027  0.041599296 -0.233776386 -0.238698805
## [50,]  0.055066113 -0.052575093 -0.224554351 -0.020686146  0.041204920
##               PC26          PC27          PC28         PC29          PC30
##  [1,]  0.025367963  0.0299650628  0.1871501825 -0.203545990 -0.1439739810
##  [2,] -0.026468725  0.0481811725  0.1473375393  0.064221896  0.0115245126
##  [3,]  0.179360071 -0.2557520213 -0.2435446400  0.023025231 -0.2052952858
##  [4,] -0.222418483  0.1598853672  0.0023668314  0.111506089  0.0001979637
##  [5,] -0.231647953  0.0730375856 -0.0620220108 -0.054956536 -0.1387723992
##  [6,] -0.012509613  0.0133065331 -0.0757919423 -0.076385009 -0.0366922570
##  [7,] -0.057961467 -0.1477203287  0.0870570260  0.074252695 -0.2582008101
##  [8,] -0.243293566 -0.0870003513 -0.0606707815 -0.083123721  0.1667624894
##  [9,]  0.162765449 -0.1199301645  0.3539049124 -0.050277645 -0.0116482529
## [10,]  0.009719559 -0.0661012824  0.0662009461 -0.045343266  0.2650075300
## [11,] -0.120010544 -0.0679859983  0.1662967865  0.127092203  0.0524297498
## [12,]  0.136753683 -0.0908540928  0.1989452155 -0.003419889 -0.1166289896
## [13,]  0.054012382  0.2540746695  0.0510352794  0.082987275  0.1281455238
## [14,]  0.033579936  0.0245615373 -0.0924013444 -0.218459174  0.1509471908
## [15,] -0.117699300 -0.1102853846 -0.2078320888  0.154615748 -0.0969929478
## [16,] -0.055315954 -0.0601512196 -0.1664849572  0.023934017 -0.2845379619
## [17,] -0.051807505  0.1211793369  0.1865461808  0.191060135 -0.1632467215
## [18,] -0.254179659  0.1191486431  0.0432764288  0.187444045  0.1035911316
## [19,] -0.123870967  0.1615345453 -0.0912595787  0.002954691  0.0337063459
## [20,] -0.056490567  0.1365167709 -0.0143596326  0.018336395 -0.0478474695
## [21,]  0.083293927  0.0062801475  0.1497965811 -0.088906125  0.0808526298
## [22,]  0.005804453  0.0829880366  0.0567081176  0.138715035 -0.0119358782
## [23,] -0.087305038  0.0002071189 -0.0686014524 -0.106544860 -0.1821136966
## [24,]  0.183514233  0.0187857975 -0.1096080875 -0.021706778  0.2019566032
## [25,] -0.178958027 -0.1370068844  0.0043290505 -0.177205967  0.0004602244
## [26,] -0.377894446  0.0674759854 -0.0817860135 -0.086038168  0.1045211571
## [27,] -0.060041888 -0.2073383598  0.0235427937  0.189257195  0.0301236083
## [28,]  0.130269028 -0.0477515873  0.0545977683 -0.244268047  0.1657371881
## [29,]  0.174013566  0.2027075674 -0.0634603669  0.189101406 -0.3096960029
## [30,]  0.013287558 -0.1531293392  0.0100837553 -0.121641760 -0.0521058253
## [31,]  0.012649513 -0.0278506506  0.0250464458  0.076824091 -0.0043716967
## [32,]  0.094591323  0.1108380124 -0.1205713469  0.283164920 -0.1431019170
## [33,]  0.308380653 -0.0357237853 -0.1876093418  0.189535450  0.3430269447
## [34,]  0.016293614 -0.2114297701  0.1623394347  0.200565654 -0.0534809803
## [35,]  0.015475085 -0.0444721744  0.0827725718 -0.049563573  0.0825328926
## [36,]  0.141974323 -0.0335459251  0.0007180346 -0.108775714  0.1039086511
## [37,]  0.042261787  0.3286492397 -0.0339597400  0.200235177  0.2030216899
## [38,]  0.184576517  0.2552260257 -0.0797403151  0.003364961 -0.0307765939
## [39,]  0.026012999 -0.0273519839 -0.0280109247 -0.104306504 -0.0014692534
## [40,] -0.160976325 -0.0725345495  0.0235207881 -0.156225762 -0.0153873729
## [41,] -0.154701540 -0.1646408054 -0.2138892686  0.192823771  0.2049779347
## [42,] -0.025224591 -0.0212701927  0.2547304990 -0.132711383 -0.0399473477
## [43,]  0.204570271  0.2123094129 -0.2881693454 -0.196220238 -0.1801007431
## [44,]  0.128437173 -0.3511147559 -0.0748105858  0.152517514 -0.0701092522
## [45,]  0.079491167  0.1856489728  0.0260689209 -0.319507260 -0.1681598880
## [46,] -0.026353498 -0.1906506371 -0.1378937127 -0.071493483  0.0900918633
## [47,]  0.074449351  0.0643286331  0.3959127762  0.144022609  0.0322774369
## [48,]  0.079319047 -0.1268027937 -0.1135238063 -0.063764251  0.0041153531
## [49,] -0.076235539  0.1234068952 -0.0321646317 -0.155172490  0.0933246930
## [50,]  0.255249211 -0.0289501121  0.0129539035  0.003194214  0.1345008387
##               PC31          PC32         PC33          PC34         PC35
##  [1,] -0.185966454  0.0587712208  0.154023000  0.2352545267 -0.026204167
##  [2,]  0.124000494 -0.0690460558  0.087952057  0.0450448245  0.021481759
##  [3,]  0.062918504 -0.3226853992  0.010401240 -0.0722947620  0.012317698
##  [4,] -0.253683689 -0.3416264826 -0.073270901  0.0007493148  0.028608853
##  [5,] -0.155444085 -0.0385682921  0.160608122  0.1974336513 -0.151102325
##  [6,] -0.157562402 -0.1524306098  0.065650942 -0.0385801348  0.010924734
##  [7,] -0.120131905 -0.0112489245  0.008017331  0.1436498378 -0.175012450
##  [8,]  0.219743627  0.0685144832  0.001163075  0.0801186767  0.352830234
##  [9,] -0.023886320  0.0519690063  0.145200205  0.0355546575  0.181082022
## [10,] -0.052007866  0.1076485961 -0.050506528  0.1891379465 -0.165775342
## [11,]  0.019469780 -0.0823537738 -0.050061239  0.0977812545 -0.013836979
## [12,]  0.129362626  0.0930301550 -0.299876083 -0.1659192074  0.007847047
## [13,]  0.107430646 -0.0221531994  0.313555693  0.0548534897  0.074379205
## [14,] -0.070789672 -0.1999462932 -0.081217070 -0.1416619587 -0.161859651
## [15,] -0.108616623  0.0742753493 -0.011516352  0.0084739923 -0.021729287
## [16,]  0.118118183  0.2397789399 -0.257878970  0.1486602729 -0.230452860
## [17,]  0.041423945 -0.0962868155  0.203423088  0.1383780398 -0.010139604
## [18,]  0.041962656  0.1119344732 -0.105995594 -0.0105091389  0.159032297
## [19,] -0.045172253  0.2668938371  0.038847997 -0.1878588348  0.137759167
## [20,]  0.021589811 -0.0535495080 -0.074639099 -0.0487502044 -0.079078288
## [21,]  0.155536344 -0.1509180670 -0.087587689  0.1568947477  0.068776736
## [22,] -0.131242826 -0.0358640826 -0.190321128  0.1311802029  0.131016187
## [23,]  0.166820454 -0.0114154909 -0.224694630  0.1974451418  0.311852676
## [24,] -0.127004567 -0.1271525319 -0.106967193 -0.0019097412  0.257091486
## [25,] -0.060305941  0.1595630983  0.045794035  0.1102615270  0.032212782
## [26,] -0.077081587  0.0254833616 -0.121503336 -0.0480109424 -0.108352585
## [27,] -0.073866125  0.0370361435  0.011649133  0.0947337668 -0.118181429
## [28,]  0.202467177  0.0145415045 -0.094828646  0.1567103282  0.018740195
## [29,] -0.111171682  0.0616009823 -0.055385158 -0.0458540428  0.142739692
## [30,]  0.209154668  0.2313521084  0.168136696  0.0058555552 -0.168818517
## [31,] -0.185384829  0.3428478116  0.132557219  0.0665947078  0.190413633
## [32,]  0.179707801 -0.0565990822  0.053177451 -0.0183661687 -0.104172681
## [33,] -0.109629520  0.1393824800 -0.051844776  0.1275819775 -0.167367907
## [34,] -0.098692102 -0.1342991262 -0.037944292 -0.2955076518 -0.001600701
## [35,] -0.024563814 -0.1126517366 -0.144004960 -0.2293008722 -0.045964184
## [36,] -0.016715681 -0.0776167361 -0.199953673 -0.0026803150  0.042105696
## [37,]  0.057262114  0.0262030646  0.056530824 -0.0875489277  0.085672811
## [38,]  0.265449855  0.2061999472 -0.070603842  0.0572433112 -0.147012626
## [39,] -0.297722904  0.3224058877 -0.062120762 -0.2666381922 -0.071374995
## [40,]  0.170211064 -0.0985901048 -0.213531762 -0.0202064179 -0.057897338
## [41,]  0.269162599 -0.0968249362  0.220422072 -0.0086087062 -0.230464434
## [42,] -0.034250853 -0.0147401790  0.125353268 -0.0090589121  0.008196382
## [43,]  0.003447964  0.0093480218  0.001437967  0.0337795303  0.078963536
## [44,]  0.088811226 -0.0006105085  0.186053571  0.0258619003  0.303732071
## [45,]  0.026284471 -0.0917414380  0.236523721 -0.2214995016 -0.091018276
## [46,] -0.149459026  0.1114370314  0.152588122 -0.2828869525  0.117400178
## [47,]  0.125087178  0.1012452051 -0.072342433 -0.2442070633 -0.213789879
## [48,]  0.099478127 -0.0011752173  0.242268215 -0.0886808627 -0.030040566
## [49,]  0.019437250 -0.0896457355  0.192129623  0.0207463346 -0.067752940
## [50,] -0.304480641 -0.0657439752 -0.023850841  0.3467536576 -0.144194827
##               PC36         PC37        PC38         PC39          PC40
##  [1,] -0.113013692 -0.013635731  0.09217627  0.190039572  0.1062860544
##  [2,]  0.118544630 -0.116581952  0.02403489  0.167886756 -0.0009604968
##  [3,] -0.040744755 -0.153936274 -0.20230334 -0.046370925  0.1369202366
##  [4,]  0.019507809 -0.082922504  0.01328199 -0.254467211 -0.1402474466
##  [5,] -0.242201752  0.112875025 -0.08139808 -0.003409105  0.0046964470
##  [6,]  0.113870008  0.105007827  0.13755287  0.197601944  0.1871432733
##  [7,] -0.039781642  0.084611873 -0.05443410  0.094552084  0.1364491148
##  [8,]  0.087387385  0.050684770 -0.06915648 -0.050695877  0.0235304060
##  [9,] -0.181560650 -0.042673178  0.27882739 -0.001683822 -0.1518375648
## [10,]  0.040382177 -0.059985250  0.09175950  0.009493088 -0.3583843870
## [11,]  0.141459545 -0.266333926 -0.19180723 -0.093691214  0.2982573861
## [12,]  0.148729831  0.074520337  0.22392174  0.025806641  0.2851843914
## [13,]  0.019450551 -0.202154884 -0.02040641  0.093975467  0.0120330135
## [14,]  0.174535727 -0.268545161  0.13096323  0.167243612 -0.0797643375
## [15,]  0.140890457 -0.072393973 -0.16397571  0.138378646 -0.2772527864
## [16,] -0.131153854 -0.007107414  0.06582871  0.034428000 -0.2849406887
## [17,]  0.355011128  0.169913381  0.04389148 -0.221897471  0.0662587400
## [18,]  0.067862245 -0.067313657  0.06180857  0.125561308  0.0647318918
## [19,] -0.227807321  0.033380228  0.12496315  0.009400515  0.0506958420
## [20,] -0.054535841  0.214241729 -0.03521927 -0.118242668  0.0639191546
## [21,] -0.156538989 -0.100671487 -0.03120757 -0.061556798 -0.1943963562
## [22,] -0.201411193  0.067541234 -0.14770706 -0.031058946  0.1573872849
## [23,] -0.031809111 -0.097563070  0.22952378 -0.059707167  0.0665239548
## [24,]  0.046627885  0.039306499  0.18295749  0.022564752  0.1385934552
## [25,] -0.029782692  0.026565224 -0.07593412 -0.250676291 -0.0132750235
## [26,] -0.033162604 -0.112642753  0.09227188  0.210395897  0.1053872362
## [27,]  0.084085630  0.298189435  0.13388570  0.146980280  0.0368934660
## [28,]  0.017706953  0.216153079 -0.47517442 -0.050702989  0.0329790809
## [29,]  0.098091029 -0.092023967 -0.10559934  0.117367970 -0.0888917560
## [30,]  0.164696156 -0.285898318 -0.06779599 -0.228521795  0.1142755972
## [31,]  0.084868827 -0.134049493 -0.14425716 -0.007787922  0.0073646424
## [32,] -0.049973106 -0.074381649  0.14642554 -0.049958847 -0.0495417293
## [33,] -0.147419922 -0.101641374  0.09534540 -0.123605712  0.1055643685
## [34,] -0.165388356 -0.070075909 -0.16591523 -0.177994176 -0.0523615288
## [35,] -0.128890331  0.272000997 -0.09063547 -0.018990467 -0.0698617910
## [36,]  0.244555414  0.206414611  0.03383012  0.102487375 -0.1440422509
## [37,] -0.173765794  0.122209016 -0.24937079  0.088320994 -0.1082278622
## [38,]  0.040107531  0.049604924 -0.10865709  0.114080793  0.1421357704
## [39,]  0.257610260  0.148884046 -0.08588752 -0.114068699  0.0511252769
## [40,] -0.126036330 -0.184196479 -0.07009040  0.116007922 -0.0433308533
## [41,]  0.008384181  0.177381498  0.23084388 -0.116788690  0.0431519429
## [42,] -0.191978020  0.123434747  0.08197953 -0.114293898 -0.0306562983
## [43,] -0.063951572  0.041618486  0.20424973 -0.405384828 -0.0314683768
## [44,]  0.087469165  0.135660956  0.00344335  0.180795954 -0.2824816925
## [45,]  0.093234999 -0.026639697 -0.08095032  0.169048914 -0.1440442005
## [46,] -0.098236984 -0.189499310  0.01250810 -0.119613798  0.0222531145
## [47,] -0.059347862 -0.089845748  0.07037844 -0.071590974 -0.0864479137
## [48,] -0.301721885  0.077777024 -0.01878256  0.257385667  0.2738477690
## [49,]  0.185584281  0.133937209  0.01624358 -0.141018243 -0.0562864337
## [50,]  0.022771885 -0.110529012 -0.02872197  0.028976111  0.0685660610
##               PC41         PC42          PC43         PC44         PC45
##  [1,] -0.053532344 -0.210610375 -0.0351752450  0.188214797  0.102981260
##  [2,]  0.067677687  0.008622570 -0.0075606163  0.096417905  0.176078738
##  [3,]  0.071045605  0.130513502  0.0103526120  0.111603513 -0.198168374
##  [4,]  0.204994465  0.158386575 -0.0810254454 -0.028344489 -0.037544400
##  [5,] -0.065824306  0.284249638 -0.0113005929 -0.103424054 -0.120589668
##  [6,] -0.066587237  0.103238314 -0.3200999122  0.102892726 -0.151461705
##  [7,] -0.040355961 -0.027900699 -0.0008583171 -0.210340899  0.149245357
##  [8,]  0.022081691  0.123524291  0.0089184515 -0.115473517  0.075789984
##  [9,]  0.276215189  0.087381781  0.1079463864 -0.213834302 -0.226502499
## [10,] -0.175736995  0.012881988  0.2051275052 -0.210056270 -0.028992135
## [11,] -0.111888125  0.091224155  0.0043722850 -0.258781990 -0.058247496
## [12,]  0.151579126  0.166336195  0.0256465604 -0.229854274 -0.042259408
## [13,] -0.003117017  0.152048241 -0.1661047905  0.108637611  0.044205139
## [14,]  0.087473158 -0.274933484  0.1473364127 -0.003094131 -0.038574600
## [15,]  0.071116432  0.093360014  0.1274229457 -0.097674707 -0.008136812
## [16,]  0.047810336  0.094479658  0.0355539955  0.307116273 -0.107330762
## [17,] -0.092124437  0.027587461  0.2274570185  0.198446465  0.017456714
## [18,]  0.105901783 -0.075097751  0.0593547060  0.108057744 -0.180498499
## [19,] -0.028845103  0.040299642 -0.1941715501 -0.133489124  0.185225986
## [20,]  0.181059670 -0.070017423  0.0230705158 -0.072131291  0.402233711
## [21,] -0.142876728  0.200590053  0.0903593481 -0.053427101  0.108777105
## [22,]  0.045820920 -0.335594204 -0.0253846199 -0.084848162 -0.329278041
## [23,] -0.204704701 -0.175001645 -0.0216204603  0.184881392  0.140516562
## [24,] -0.079033825  0.081000764  0.1229381704  0.172886254  0.035599093
## [25,]  0.333891133  0.078196486 -0.2398908587  0.110006425  0.010873552
## [26,]  0.134418461 -0.009526182  0.1558366914 -0.078573261  0.082060512
## [27,] -0.110397596  0.085545702  0.1225721663  0.194428357 -0.107866838
## [28,]  0.071847118  0.006415378  0.1579184052 -0.007472906  0.123727232
## [29,] -0.068668720 -0.036498340  0.2691871347 -0.056196254  0.219594062
## [30,] -0.075939235 -0.013625100 -0.1491369261  0.118151309 -0.041530255
## [31,] -0.068008060 -0.082121527 -0.0164438444  0.051115324 -0.248262623
## [32,]  0.020038568  0.120647161 -0.2165434903 -0.057216667  0.230256621
## [33,]  0.109104587  0.043548125 -0.0766981839  0.101493345  0.054656412
## [34,] -0.094543598 -0.245019775  0.0342528697  0.114214462  0.031999383
## [35,]  0.131396740 -0.044095441 -0.1018541563  0.150613586  0.014745354
## [36,] -0.142111134  0.284451759 -0.1845048393  0.109655277 -0.091406634
## [37,] -0.220641610 -0.012279540 -0.0668962194  0.053485575 -0.127173002
## [38,]  0.321100370 -0.122949767  0.0544983233  0.054117482 -0.113196274
## [39,] -0.262083248 -0.001117219 -0.1078637728 -0.132553792  0.094773522
## [40,] -0.332884169 -0.113573338 -0.2381514791 -0.019805918  0.007990628
## [41,] -0.078453091 -0.282756911  0.0179709073 -0.142391372 -0.083648167
## [42,] -0.068391573 -0.096999948  0.0754006239  0.186025657  0.064250615
## [43,] -0.163784546 -0.056733516  0.0869985391 -0.249578617 -0.190456073
## [44,]  0.109363451 -0.228886407 -0.2577474302 -0.178569968  0.115063151
## [45,]  0.090468436 -0.042187929 -0.0684979762 -0.094133782 -0.128272498
## [46,]  0.067564037  0.112919940  0.2703317873  0.235180404  0.172664099
## [47,] -0.107717699  0.082910245 -0.0725515921  0.038428164 -0.036440286
## [48,] -0.160190968  0.185623058  0.2514272284 -0.081652531  0.006723520
## [49,]  0.013281793 -0.157649250  0.0675264668  0.049904304  0.004325586
## [50,]  0.033667643 -0.088438913 -0.1688821446 -0.069464701  0.228687721
##               PC46         PC47         PC48         PC49         PC50
##  [1,] -0.211152765 -0.037502977 -0.130751686  0.009228641  0.111816555
##  [2,]  0.061283694 -0.184792831 -0.422257331  0.073142515  0.193676012
##  [3,] -0.102783475 -0.023110172 -0.067310386 -0.024832499  0.028895396
##  [4,]  0.202494862 -0.037323340  0.061690014  0.232402002  0.062710618
##  [5,] -0.102519022 -0.023900289 -0.247365007 -0.190617493 -0.062353760
##  [6,]  0.032661129 -0.139202391  0.325455462  0.153384514  0.055034702
##  [7,]  0.189378937 -0.182385657 -0.130186993  0.090387808 -0.163477202
##  [8,]  0.087819657 -0.047150813  0.216480336  0.036793158 -0.087711959
##  [9,]  0.065173887  0.174669320  0.040392729 -0.341081590 -0.143162062
## [10,] -0.113988149 -0.080379422  0.101765909  0.134545036 -0.051449198
## [11,] -0.001086310 -0.179031633 -0.026311378 -0.110532955 -0.036126920
## [12,] -0.041924138 -0.120610996 -0.012644136  0.112104159  0.126210027
## [13,]  0.283871690 -0.062442876  0.008394039 -0.266649278  0.005028121
## [14,] -0.004558391 -0.278120641 -0.091157533 -0.091324760 -0.043094750
## [15,] -0.120243173  0.140449936  0.040958832 -0.024916900  0.099110725
## [16,]  0.250027234 -0.226295828  0.112905080  0.021895662 -0.095108259
## [17,] -0.273485316  0.119574944  0.043837992  0.122640773 -0.098231609
## [18,] -0.100595112  0.079939777  0.016773535  0.060081686 -0.187309522
## [19,]  0.031938298  0.021862025 -0.132355867  0.298599168  0.113541484
## [20,] -0.222234779 -0.052327381  0.049957445 -0.018984351 -0.070115544
## [21,] -0.050825453  0.198624331 -0.245542774  0.330849621 -0.135216869
## [22,] -0.006161111  0.081310409 -0.140733270  0.003563248  0.041127632
## [23,]  0.137611003  0.043951374  0.009861859 -0.042196039 -0.093466610
## [24,] -0.072916204  0.036500346  0.002975002 -0.170397366 -0.035269084
## [25,] -0.256720184 -0.156530604  0.169086617 -0.138046792 -0.048895259
## [26,]  0.051918964  0.296741409 -0.005156454 -0.083009678  0.216571494
## [27,]  0.084359415  0.100688020 -0.114227791 -0.148738103  0.208138612
## [28,]  0.120369258 -0.077344549  0.058605865 -0.118307810  0.369882158
## [29,] -0.017476550  0.098581297  0.290640171 -0.123374300  0.136778131
## [30,]  0.071155368  0.358980614 -0.045365747  0.098313101  0.044415991
## [31,]  0.079238872  0.012357758 -0.006895099  0.168388181  0.232763624
## [32,] -0.028086257  0.128976176 -0.048914093 -0.244693795  0.065478758
## [33,] -0.313425032  0.017200560  0.025756825  0.056665381 -0.019064088
## [34,]  0.090324161  0.036390681  0.048763779  0.079261251 -0.182774450
## [35,]  0.194667953  0.343389070 -0.085604404 -0.057083611  0.058041118
## [36,] -0.013416299  0.053232165 -0.155293980  0.083629071 -0.035955754
## [37,] -0.072130518 -0.193744109 -0.012539500 -0.049838277 -0.049560311
## [38,]  0.044999381  0.011204272 -0.103756290  0.092586309 -0.287551306
## [39,]  0.061150489  0.024125475 -0.142951631 -0.229385150 -0.256943121
## [40,] -0.265377655  0.097062065  0.093014215 -0.213699531 -0.022636709
## [41,]  0.051439416  0.009663853  0.055125194  0.024795029  0.172635139
## [42,] -0.041059823 -0.134853410  0.172312932  0.039954393  0.160704027
## [43,]  0.067628298 -0.078310239 -0.096919879 -0.031034375  0.214601213
## [44,] -0.054041292 -0.115630361 -0.103497075  0.043803861 -0.030808591
## [45,] -0.156878132  0.157100915  0.094065543  0.172780733  0.036943326
## [46,]  0.028831197 -0.151257218 -0.195309950  0.016718701  0.010317848
## [47,]  0.044768598 -0.170529642  0.151432981 -0.035378367  0.155381862
## [48,]  0.096404055  0.038747503  0.230939224  0.159622208 -0.219161511
## [49,]  0.222727430 -0.010315626 -0.029405712 -0.091717767 -0.272315390
## [50,]  0.281934035  0.167270306  0.204337433  0.087660553 -0.033825316
biplot(pca.x, col = c(2, 4))

Cols = function(vec){
  cols = rainbow(length(unique(vec)))
  + return (cols[as.numeric(as.factor(vec))])
  }

plot(pca.x$x[,1:2],col = Cols(k_col), pch =19, xlab ="Z1", ylab =" Z2")

#(c)
km.k3 <- kmeans(x,3,nstart = 100)
km.k3
## K-means clustering with 3 clusters of sizes 20, 20, 20
## 
## Cluster means:
##       [,1]      [,2]      [,3]      [,4]      [,5]     [,6]      [,7]
## 1 1.309855 1.5297578 1.1779362 1.0127173 1.1392161 1.126534 1.2565549
## 2 2.475264 2.2168702 2.9349253 2.4655435 2.1325087 2.614003 1.9749243
## 3 0.649094 0.4272122 0.3176989 0.2322788 0.3968267 0.466004 0.6723788
##        [,8]      [,9]     [,10]     [,11]     [,12]    [,13]     [,14]
## 1 1.2520303 1.1322457 0.8168329 1.1008661 1.0163088 1.207543 1.1516678
## 2 2.5297753 2.2319161 2.7398656 2.2545316 2.6359340 2.547966 2.6918660
## 3 0.3036237 0.2740587 0.6096622 0.6082911 0.3533017 0.862027 0.3103367
##       [,15]     [,16]     [,17]     [,18]       [,19]     [,20]     [,21]
## 1 1.1807631 1.0831205 1.0491036 1.3404711  0.95348547 0.1843175 1.2996476
## 2 2.3339909 2.4653947 2.1422365 2.7265256  2.07489790 2.8289318 2.4326758
## 3 0.5646508 0.6014428 0.6664156 0.5242812 -0.02815256 0.7511646 0.3753776
##      [,22]     [,23]     [,24]     [,25]     [,26]     [,27]     [,28]
## 1 1.025534 1.4607360 0.9940398 1.0590802 1.1322356 1.0087072 1.1361883
## 2 2.388767 1.9959622 2.3270780 2.2484951 2.2870269 2.5202806 2.3146460
## 3 0.444185 0.5862425 0.4881267 0.2986866 0.4242547 0.6580425 0.5569209
##       [,29]     [,30]     [,31]    [,32]     [,33]     [,34]     [,35]
## 1 1.7040022 1.3632733 0.8239648 1.320278 1.5653810 0.9339115 1.0138445
## 2 2.5695387 2.3448948 2.2700875 2.628201 1.9682686 2.5716311 2.2965644
## 3 0.1579515 0.7280993 1.0807349 1.055913 0.3004261 0.3584250 0.7454241
##       [,36]     [,37]     [,38]     [,39]     [,40]       [,41]     [,42]
## 1 1.2694280 1.2306369 1.1967791 1.6208508 1.1248914  1.27757363 1.4278667
## 2 2.4356080 2.3580215 2.5349614 2.6495192 2.6114833  1.99571553 2.1598609
## 3 0.6046648 0.3827459 0.3997692 0.5030772 0.5984623 -0.07758204 0.2804351
##       [,43]     [,44]     [,45]     [,46]     [,47]     [,48]     [,49]
## 1 0.9946385 1.1585822 1.3213006 1.0960575 1.4631512 1.1495011 1.2320374
## 2 2.2230252 2.4162878 2.6572722 2.1466762 2.0968924 2.2132917 2.2889500
## 3 0.6423291 0.7560256 0.3578973 0.8022603 0.5809075 0.6496547 0.2696592
##       [,50]
## 1 1.2560154
## 2 2.0582427
## 3 0.7619741
## 
## Clustering vector:
##  [1] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 1 3 3 3 3 3
## [36] 3 3 3 3 3 1 1 1 1 1 1 1 3 1 1 1 1 1 1 1 1 1 1 1 1
## 
## Within cluster sum of squares by cluster:
## [1] 877.2126 902.4400 935.7357
##  (between_SS / total_SS =  41.5 %)
## 
## Available components:
## 
## [1] "cluster"      "centers"      "totss"        "withinss"    
## [5] "tot.withinss" "betweenss"    "size"         "iter"        
## [9] "ifault"
km.k3.table <- table(k,km.k3$cluster)
km.k3.table
##            
## k            1  2  3
##   CLUSTER A  0 20  0
##   CLUSTER B  1  0 19
##   CLUSTER C 19  0  1
#The row labels are the true cluster assignments for each observation and the columns labels are the kmean cluster asignments.
#In this format correct kmeans clustering assignment would show only one 20 value in any row or column. There for we can see
#that there are 2 errors in the kmeans clustering assignment.

#(d)
km.k2 <- kmeans(x,2,nstart = 100)
km.k2
## K-means clustering with 2 clusters of sizes 40, 20
## 
## Cluster means:
##        [,1]     [,2]      [,3]      [,4]      [,5]      [,6]      [,7]
## 1 0.9794745 0.978485 0.7478175 0.6224981 0.7680214 0.7962691 0.9644669
## 2 2.4752637 2.216870 2.9349253 2.4655435 2.1325087 2.6140035 1.9749243
##       [,8]      [,9]     [,10]     [,11]     [,12]    [,13]     [,14]
## 1 0.777827 0.7031522 0.7132475 0.8545786 0.6848053 1.034785 0.7310023
## 2 2.529775 2.2319161 2.7398656 2.2545316 2.6359340 2.547966 2.6918660
##      [,15]     [,16]     [,17]     [,18]     [,19]    [,20]     [,21]
## 1 0.872707 0.8422817 0.8577596 0.9323762 0.4626665 0.467741 0.8375126
## 2 2.333991 2.4653947 2.1422365 2.7265256 2.0748979 2.828932 2.4326758
##       [,22]    [,23]     [,24]     [,25]     [,26]     [,27]     [,28]
## 1 0.7348595 1.023489 0.7410833 0.6788834 0.7782452 0.8333748 0.8465546
## 2 2.3887665 1.995962 2.3270780 2.2484951 2.2870269 2.5202806 2.3146460
##       [,29]    [,30]     [,31]    [,32]     [,33]     [,34]     [,35]
## 1 0.9309769 1.045686 0.9523499 1.188096 0.9329035 0.6461682 0.8796343
## 2 2.5695387 2.344895 2.2700875 2.628201 1.9682686 2.5716311 2.2965644
##       [,36]     [,37]     [,38]    [,39]     [,40]     [,41]     [,42]
## 1 0.9370464 0.8066914 0.7982742 1.061964 0.8616768 0.5999958 0.8541509
## 2 2.4356080 2.3580215 2.5349614 2.649519 2.6114833 1.9957155 2.1598609
##       [,43]     [,44]     [,45]     [,46]    [,47]     [,48]     [,49]
## 1 0.8184838 0.9573039 0.8395989 0.9491589 1.022029 0.8995779 0.7508483
## 2 2.2230252 2.4162878 2.6572722 2.1466762 2.096892 2.2132917 2.2889500
##      [,50]
## 1 1.008995
## 2 2.058243
## 
## Clustering vector:
##  [1] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
## [36] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
## 
## Within cluster sum of squares by cluster:
## [1] 2101.808  902.440
##  (between_SS / total_SS =  35.3 %)
## 
## Available components:
## 
## [1] "cluster"      "centers"      "totss"        "withinss"    
## [5] "tot.withinss" "betweenss"    "size"         "iter"        
## [9] "ifault"
km.k2.table <- table(k,km.k2$cluster)
km.k2.table
##            
## k            1  2
##   CLUSTER A  0 20
##   CLUSTER B 20  0
##   CLUSTER C 20  0
#This time the kmeans clustering function has completely clustered together 2 of the 3 original clusters that have the closest
#mean speration [1.2-0.5=0.7].

#(e)
km.k4 <- kmeans(x,4,nstart = 100)
km.k4
## K-means clustering with 4 clusters of sizes 11, 19, 20, 10
## 
## Cluster means:
##        [,1]      [,2]      [,3]       [,4]       [,5]       [,6]      [,7]
## 1 0.9684234 0.3104380 0.2573813  0.8701532  0.9640425 0.08611265 0.6398039
## 2 1.3390477 1.5762877 1.2286939  1.0778906  1.1629171 1.13801962 1.2230211
## 3 2.4752637 2.2168702 2.9349253  2.4655435  2.1325087 2.61400346 1.9749243
## 4 0.3084418 0.5775117 0.3736324 -0.5151685 -0.1979035 0.92811533 0.8303432
##         [,8]      [,9]     [,10]     [,11]     [,12]     [,13]     [,14]
## 1 0.77248789 0.3738488 0.2310056 0.6569641 0.4477177 0.8078905 0.4290859
## 2 1.17057978 1.1047955 0.9317316 1.1470023 1.0751547 1.2454097 1.1125388
## 3 2.52977527 2.2319161 2.7398656 2.2545316 2.6359340 2.5479659 2.6918660
## 4 0.03746968 0.3022635 0.8285940 0.5163493 0.2039378 0.8841818 0.3381909
##       [,15]     [,16]     [,17]     [,18]      [,19]     [,20]     [,21]
## 1 0.6605928 0.2689090 0.4100315 0.8345475 -0.1321308 0.8841748 0.3051126
## 2 1.1961687 1.1182988 1.0402508 1.3524986  0.9878381 0.1707013 1.3106438
## 3 2.3339909 2.4653947 2.1422365 2.7265256  2.0748979 2.8289318 2.4326758
## 4 0.4914554 0.9485591 1.0035271 0.2417551  0.1191174 0.5740394 0.5242033
##       [,22]     [,23]     [,24]     [,25]      [,26]      [,27]     [,28]
## 1 0.3068591 0.9625196 0.3111984 0.1862789 -0.4517381 1.17819360 0.6572198
## 2 1.0585594 1.5253206 1.0313083 1.0992041  1.2273931 1.04686810 1.1128662
## 3 2.3887665 1.9959622 2.3270780 2.2484951  2.2870269 2.52028064 2.3146460
## 4 0.5906302 0.1370763 0.6625290 0.4221391  1.2778458 0.04843702 0.5488309
##        [,29]     [,30]     [,31]     [,32]     [,33]     [,34]     [,35]
## 1  0.6278806 0.8743944 1.0587433 0.8341859 0.5258548 0.2737157 0.6620560
## 2  1.7859342 1.3869036 0.8067553 1.3504132 1.6141058 0.9800260 1.0421817
## 3  2.5695387 2.3448948 2.2700875 2.6282012 1.9682686 2.5716311 2.2965644
## 4 -0.3600361 0.5857944 1.1119467 1.2689925 0.0863730 0.4215364 0.8101305
##       [,36]     [,37]       [,38]     [,39]    [,40]      [,41]     [,42]
## 1 0.9099156 0.4114976  0.85664455 0.6038263 0.815035  0.5345661 0.0801116
## 2 1.1978760 1.2409721  1.18988905 1.5986688 1.126306  1.2404264 1.3640089
## 3 2.4356080 2.3580215  2.53496138 2.6495192 2.611483  1.9957155 2.1598609
## 4 0.4713142 0.4162712 -0.01000155 0.5461764 0.410188 -0.5448498 0.7368640
##       [,43]     [,44]       [,45]    [,46]       [,47]     [,48]
## 1 0.6388868 0.7168863  0.83765127 1.109522 1.186103308 0.5836007
## 2 0.9769967 1.2057861  1.33073984 1.096371 1.463939417 1.2272089
## 3 2.2230252 2.4162878  2.65727223 2.146676 2.096892407 2.2132917
## 4 0.7148658 0.7496473 -0.09142632 0.493057 0.001919018 0.6246537
##        [,49]     [,50]
## 1 0.08989182 1.2203810
## 2 1.33194302 1.1780466
## 3 2.28895004 2.0582427
## 4 0.37382039 0.4552712
## 
## Clustering vector:
##  [1] 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 1 4 4 1 1 1 4 4 1 1 4 1 4 4 1
## [36] 4 1 4 1 4 2 2 2 2 2 2 2 1 2 2 2 2 2 2 2 2 2 2 2 2
## 
## Within cluster sum of squares by cluster:
## [1] 463.2582 834.2581 902.4400 414.4985
##  (between_SS / total_SS =  43.7 %)
## 
## Available components:
## 
## [1] "cluster"      "centers"      "totss"        "withinss"    
## [5] "tot.withinss" "betweenss"    "size"         "iter"        
## [9] "ifault"
km.k4.table <- table(k,km.k4$cluster)
km.k4.table
##            
## k            1  2  3  4
##   CLUSTER A  0  0 20  0
##   CLUSTER B 10  0  0 10
##   CLUSTER C  1 19  0  0
#This time the kmeans clustering function has divided cluster B into two new clusters.

#(f)
km.pca.k3 <- kmeans(pca.x$x[,1:2],3,nstart = 100)
km.pca.k3
## K-means clustering with 3 clusters of sizes 21, 20, 19
## 
## Cluster means:
##         PC1        PC2
## 1 -1.410705 -0.8315563
## 2  7.366597  0.3233194
## 3 -6.195112  0.5787523
## 
## Clustering vector:
##  [1] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 1 3 3 3 3 3
## [36] 3 3 3 3 3 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
## 
## Within cluster sum of squares by cluster:
## [1] 73.25643 86.38012 55.48136
##  (between_SS / total_SS =  89.7 %)
## 
## Available components:
## 
## [1] "cluster"      "centers"      "totss"        "withinss"    
## [5] "tot.withinss" "betweenss"    "size"         "iter"        
## [9] "ifault"
km.pca.k3.table <- table(k,km.pca.k3$cluster)
km.pca.k3.table
##            
## k            1  2  3
##   CLUSTER A  0 20  0
##   CLUSTER B  1  0 19
##   CLUSTER C 20  0  0
#This kmeans cluster analysis using princple components 1 and 2 only shows similar accuracy to the kmeans cluster analysis
#in a which we would expect given that all variables have the same random spread and the same mean shift.

#(g)

x.scaled <- scale(x, scale=T)

km.k3.scaled <- kmeans(x.scaled,3,nstart = 100)
km.k3.scaled
## K-means clustering with 3 clusters of sizes 21, 19, 20
## 
## Cluster means:
##         [,1]       [,2]       [,3]       [,4]        [,5]       [,6]
## 1 -0.1039736  0.0883601 -0.2754701 -0.1087234 -0.07246093 -0.2001155
## 2 -0.6993348 -0.7893349 -0.7328472 -0.7891656 -0.72699079 -0.7416277
## 3  0.7735403  0.6570901  0.9854484  0.8638669  0.76672522  0.9146676
##          [,7]        [,8]        [,9]      [,10]      [,11]      [,12]
## 1 -0.05695131 -0.06555976 -0.04070619 -0.4138295 -0.2156899 -0.2315845
## 2 -0.60788895 -0.77346682 -0.94387071 -0.5607947 -0.5513707 -0.7491591
## 3  0.63729337  0.80363122  0.93941867  0.9672759  0.7502766  0.9548648
##        [,13]      [,14]      [,15]      [,16]      [,17]      [,18]
## 1 -0.3394769 -0.1734508 -0.1033403 -0.2716986 -0.2068716 -0.1536902
## 2 -0.5618599 -0.8065449 -0.6978473 -0.6805532 -0.5788269 -0.7484202
## 3  0.8902177  0.9483410  0.7714623  0.9318091  0.7671007  0.8723739
##         [,19]      [,20]      [,21]      [,22]      [,23]      [,24]
## 1 -0.05056514 -0.6915034 -0.1343497 -0.2202581  0.1314797 -0.1842246
## 2 -0.81174696 -0.3803478 -0.7504692 -0.6819561 -0.7421851 -0.6597100
## 3  0.82425301  1.0874090  0.8540129  0.8791293  0.5670222  0.8201604
##        [,25]      [,26]      [,27]      [,28]      [,29]       [,30]
## 1 -0.1457785 -0.2204352 -0.2549129 -0.1429850  0.1016353 -0.02904591
## 2 -0.7098896 -0.5441009 -0.6354125 -0.7908161 -0.9830227 -0.76921815
## 3  0.8274626  0.7483528  0.8713004  0.9014095  0.8271545  0.76125545
##        [,31]      [,32]      [,33]      [,34]      [,35]      [,36]
## 1 -0.4780791 -0.2823916  0.2086279 -0.2456009 -0.3201466 -0.1740479
## 2 -0.2438967 -0.4444246 -0.8413057 -0.7371824 -0.4747019 -0.5893891
## 3  0.7336849  0.7187146  0.5801811  0.9582042  0.7871207  0.7426700
##         [,37]      [,38]       [,39]      [,40]      [,41]      [,42]
## 1 -0.09152871 -0.1930792 -0.01455523 -0.2337352  0.1489961 -0.0478431
## 2 -0.77311624 -0.7224607 -0.87726224 -0.7382061 -0.9377189 -0.6856784
## 3  0.83056558  0.8890708  0.84868212  0.9467178  0.7343870  0.7016297
##        [,43]      [,44]      [,45]      [,46]       [,47]      [,48]
## 1 -0.2785215 -0.2458261 -0.1029970 -0.1844529  0.05346993 -0.1668043
## 2 -0.5965281 -0.6176396 -0.8006893 -0.5417193 -0.73601746 -0.6928590
## 3  0.8591492  0.8448750  0.8688016  0.7083089  0.64307316  0.8333605
##         [,49]      [,50]
## 1 -0.06632607 -0.1318751
## 2 -0.75751895 -0.5970250
## 3  0.78928537  0.7056426
## 
## Clustering vector:
##  [1] 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2 2 2 1 2 2 2 2 2
## [36] 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
## 
## Within cluster sum of squares by cluster:
## [1] 609.3490 560.5724 576.5949
##  (between_SS / total_SS =  40.8 %)
## 
## Available components:
## 
## [1] "cluster"      "centers"      "totss"        "withinss"    
## [5] "tot.withinss" "betweenss"    "size"         "iter"        
## [9] "ifault"
km.k3.scaled.table <- table(k,km.k3.scaled$cluster)
km.k3.scaled.table
##            
## k            1  2  3
##   CLUSTER A  0  0 20
##   CLUSTER B  1 19  0
##   CLUSTER C 20  0  0
#This kmeans cluster analysis using princple components 1 and 2 only shows similar accuracy to the kmeans cluster analysis
#in a which we would expect given that all variables have the same random spread and the same mean shift.